#include "../pch.h"
#include "REFEncoding.h"
#include "REFUTF8Encoding.h"
#include "REFUTF32Encoding.h"
#include "REFUTF16Encoding.h"
#include "REFASCIIEncoding.h"
#include "REFLatin1Encoding.h"
#include "REFBaseCodePageEncoding.h"
#include "REFSBCSCodePageEncoding.h"
#include "REFDBCSCodePageEncoding.h"
#include "REFDecoderReplacementFallback.h"

#ifdef	HAVE_LANGINFO_CODESET
#include <langinfo.h>
#endif	//	HAVE_LANGINFO_CODESET

namespace BFX
{

int REFEncoding::s_nDefaultCodePage = 0;	// uninitialized.

Mutex	REFEncoding::s_syncMutex;
REF<REFEncodingMap>	REFEncoding::s_pEncodings = NULL;

REFEncoding::REFEncoding(int nPageCode) :
		m_nCodePage(nPageCode)
{
	BFX_REQUIRE(nPageCode > 0);

	m_pDecoderFallback = new REFDecoderReplacementFallback('?');
}

REFEncoding::~REFEncoding()
{
}

/////////////////////////////////////////////////////////////////////////////
// CP description table, description of most all knowing encodings.

struct CP_NAME_DESC
{
	int		CodePage;
	char	DescName[50];
};

static const CP_NAME_DESC s_CodePageDescTable []	=
{  
	{ 37, "IBM EBCDIC (US-Canada)" },
	{ 437, "OEM United States" },
	{ 500, "IBM EBCDIC (International)" },
	{ 708, "Arabic (ASMO 708)" },
	{ 720, "Arabic (DOS)" },
	{ 737, "Greek (DOS)" },
	{ 775, "Baltic (DOS)" },
	{ 850, "Western European (DOS)" },
	{ 852, "Central European (DOS)" },
	{ 855, "OEM Cyrillic" },
	{ 857, "Turkish (DOS)" },
	{ 858, "OEM Multilingual Latin I" },
	{ 860, "Portuguese (DOS)" },
	{ 861, "Icelandic (DOS)" },
	{ 862, "Hebrew (DOS)" },
	{ 863, "French Canadian (DOS)" },
	{ 864, "Arabic (864)" },
	{ 865, "Nordic (DOS)" },
	{ 866, "Cyrillic (DOS)" },
	{ 869, "Greek, Modern (DOS)" },
	{ 870, "IBM EBCDIC (Multilingual Latin-2)" },
	{ 874, "Thai (Windows)" },
	{ 875, "IBM EBCDIC (Greek Modern)" },
	{ 932, "Japanese (Shift-JIS)" },
	{ 936, "Chinese Simplified (GB2312)" },
	{ 949, "Korean" },
	{ 950, "Chinese Traditional (Big5)" },
	{ 1026, "IBM EBCDIC (Turkish Latin-5)" },
	{ 1047, "IBM Latin-1" },
	{ 1140, "IBM EBCDIC (US-Canada-Euro)" },
	{ 1141, "IBM EBCDIC (Germany-Euro)" },
	{ 1142, "IBM EBCDIC (Denmark-Norway-Euro)" },
	{ 1143, "IBM EBCDIC (Finland-Sweden-Euro)" },
	{ 1144, "IBM EBCDIC (Italy-Euro)" },
	{ 1145, "IBM EBCDIC (Spain-Euro)" },
	{ 1146, "IBM EBCDIC (UK-Euro)" },
	{ 1147, "IBM EBCDIC (France-Euro)" },
	{ 1148, "IBM EBCDIC (International-Euro)" },
	{ 1149, "IBM EBCDIC (Icelandic-Euro)" },
	{ 1200, "Unicode" },
	{ 1201, "Unicode (Big-Endian)" },
	{ 1250, "Central European (Windows)" },
	{ 1251, "Cyrillic (Windows)" },
	{ 1252, "Western European (Windows)" },
	{ 1253, "Greek (Windows)" },
	{ 1254, "Turkish (Windows)" },
	{ 1255, "Hebrew (Windows)" },
	{ 1256, "Arabic (Windows)" },
	{ 1257, "Baltic (Windows)" },
	{ 1258, "Vietnamese (Windows)" },
	{ 1361, "Korean (Johab)" },
	{ 10000, "Western European (Mac)" },
	{ 10001, "Japanese (Mac)" },
	{ 10002, "Chinese Traditional (Mac)" },
	{ 10003, "Korean (Mac)" },
	{ 10004, "Arabic (Mac)" },
	{ 10005, "Hebrew (Mac)" },
	{ 10006, "Greek (Mac)" },
	{ 10007, "Cyrillic (Mac)" },
	{ 10008, "Chinese Simplified (Mac)" },
	{ 10010, "Romanian (Mac)" },
	{ 10017, "Ukrainian (Mac)" },
	{ 10021, "Thai (Mac)" },
	{ 10029, "Central European (Mac)" },
	{ 10079, "Icelandic (Mac)" },
	{ 10081, "Turkish (Mac)" },
	{ 10082, "Croatian (Mac)" },
	{ 12000, "Unicode (UTF-32)" },
	{ 12001, "Unicode (UTF-32 Big-Endian)" },
	{ 20000, "Chinese Traditional (CNS)" },
	{ 20001, "TCA Taiwan" },
	{ 20002, "Chinese Traditional (Eten)" },
	{ 20003, "IBM5550 Taiwan" },
	{ 20004, "TeleText Taiwan" },
	{ 20005, "Wang Taiwan" },
	{ 20105, "Western European (IA5)" },
	{ 20106, "German (IA5)" },
	{ 20107, "Swedish (IA5)" },
	{ 20108, "Norwegian (IA5)" },
	{ 20127, "US-ASCII" },
	{ 20261, "T.61" },
	{ 20269, "ISO-6937" },
	{ 20273, "IBM EBCDIC (Germany)" },
	{ 20277, "IBM EBCDIC (Denmark-Norway)" },
	{ 20278, "IBM EBCDIC (Finland-Sweden)" },
	{ 20280, "IBM EBCDIC (Italy)" },
	{ 20284, "IBM EBCDIC (Spain)" },
	{ 20285, "IBM EBCDIC (UK)" },
	{ 20290, "IBM EBCDIC (Japanese katakana)" },
	{ 20297, "IBM EBCDIC (France)" },
	{ 20420, "IBM EBCDIC (Arabic)" },
	{ 20423, "IBM EBCDIC (Greek)" },
	{ 20424, "IBM EBCDIC (Hebrew)" },
	{ 20833, "IBM EBCDIC (Korean Extended)" },
	{ 20838, "IBM EBCDIC (Thai)" },
	{ 20866, "Cyrillic (KOI8-R)" },
	{ 20871, "IBM EBCDIC (Icelandic)" },
	{ 20880, "IBM EBCDIC (Cyrillic Russian)" },
	{ 20905, "IBM EBCDIC (Turkish)" },
	{ 20924, "IBM Latin-1" },
	{ 20932, "Japanese (JIS 0208-1990 and 0212-1990)" },
	{ 20936, "Chinese Simplified (GB2312-80)" },
	{ 20949, "Korean Wansung" },
	{ 21025, "IBM EBCDIC (Cyrillic Serbian-Bulgarian)" },
	{ 21027, "Ext Alpha Lowercase" },
	{ 21866, "Cyrillic (KOI8-U)" },
	{ 28591, "Western European (ISO)" },
	{ 28592, "Central European (ISO)" },
	{ 28593, "Latin 3 (ISO)" },
	{ 28594, "Baltic (ISO)" },
	{ 28595, "Cyrillic (ISO)" },
	{ 28596, "Arabic (ISO)" },
	{ 28597, "Greek (ISO)" },
	{ 28598, "Hebrew (ISO-Visual)" },
	{ 28599, "Turkish (ISO)" },
	{ 28603, "Estonian (ISO)" },
	{ 28605, "Latin 9 (ISO)" },
	{ 29001, "Europa" },
	{ 38598, "Hebrew (ISO-Logical)" },
	{ 50000, "User Defined" },
	{ 50220, "Japanese (JIS)" },
	{ 50221, "Japanese (JIS-Allow 1 byte Kana)" },
	{ 50222, "Japanese (JIS-Allow 1 byte Kana - SO/SI)" },
	{ 50225, "Korean (ISO)" },
	{ 50227, "Chinese Simplified (ISO-2022)" },
	{ 50229, "Chinese Traditional (ISO-2022)" },
	{ 50930, "IBM EBCDIC (Japanese and Japanese Katakana)" },
	{ 50931, "IBM EBCDIC (Japanese and US-Canada)" },
	{ 50933, "IBM EBCDIC (Korean and Korean Extended)" },
	{ 50935, "IBM EBCDIC (Simplified Chinese)" },
	{ 50937, "IBM EBCDIC (Traditional Chinese)" },
	{ 50939, "IBM EBCDIC (Japanese and Japanese-Latin)" },
	{ 51932, "Japanese (EUC)" },
	{ 51936, "Chinese Simplified (EUC)" },
	{ 51949, "Korean (EUC)" },
	{ 52936, "Chinese Simplified (HZ)" },
	{ 54936, "Chinese Simplified (GB18030)" },
	{ 57002, "ISCII Devanagari" },
	{ 57003, "ISCII Bengali" },
	{ 57004, "ISCII Tamil" },
	{ 57005, "ISCII Telugu" },
	{ 57006, "ISCII Assamese" },
	{ 57007, "ISCII Oriya" },
	{ 57008, "ISCII Kannada" },
	{ 57009, "ISCII Malayalam" },
	{ 57010, "ISCII Gujarati" },
	{ 57011, "ISCII Punjabi" },
	{ 65000, "Unicode (UTF-7)" },
	{ 65001, "Unicode (UTF-8)" }
};

/////////////////////////////////////////////////////////////////////////////
// Index an encoding name into an codepage in CodePageDataTable.
struct CP_ENCODING_DATA
{
	const char*	WebName;
	int		CodePage;
};

static CP_ENCODING_DATA s_pEncodingDataTable[]	=
{
	// Total Items: 455
	// encoding name, codepage.
	{ "437", 437 },
	{ "ANSI_X3.4-1968", 20127 },
	{ "ANSI_X3.4-1986", 20127 },
	// { "_autodetect", 50932 },
	// { "_autodetect_all", 50001 },
	// { "_autodetect_kr", 50949 },
	{ "arabic", 28596 },
	{ "ascii", 20127 },
	{ "ASMO-708", 708 },
	{ "Big5", 950 },
	{ "Big5-HKSCS", 950 },
	{ "CCSID00858", 858 },
	{ "CCSID00924", 20924 },
	{ "CCSID01140", 1140 },
	{ "CCSID01141", 1141 },
	{ "CCSID01142", 1142 },
	{ "CCSID01143", 1143 },
	{ "CCSID01144", 1144 },
	{ "CCSID01145", 1145 },
	{ "CCSID01146", 1146 },
	{ "CCSID01147", 1147 },
	{ "CCSID01148", 1148 },
	{ "CCSID01149", 1149 },
	{ "chinese", 936 },
	{ "cn-big5", 950 },
	{ "CN-GB", 936 },
	{ "CP00858", 858 },
	{ "CP00924", 20924 },
	{ "CP01140", 1140 },
	{ "CP01141", 1141 },
	{ "CP01142", 1142 },
	{ "CP01143", 1143 },
	{ "CP01144", 1144 },
	{ "CP01145", 1145 },
	{ "CP01146", 1146 },
	{ "CP01147", 1147 },
	{ "CP01148", 1148 },
	{ "CP01149", 1149 },
	{ "cp037", 37 },
	{ "cp1025", 21025 },
	{ "CP1026", 1026 },
	{ "cp1256", 1256 },
	{ "CP273", 20273 },
	{ "CP278", 20278 },
	{ "CP280", 20280 },
	{ "CP284", 20284 },
	{ "CP285", 20285 },
	{ "cp290", 20290 },
	{ "cp297", 20297 },
	{ "cp367", 20127 },
	{ "cp420", 20420 },
	{ "cp423", 20423 },
	{ "cp424", 20424 },
	{ "cp437", 437 },
	{ "CP500", 500 },
	{ "cp50227", 50227 },
	// { "cp50229", 50229 },
	{ "cp819", 28591 },
	{ "cp850", 850 },
	{ "cp852", 852 },
	{ "cp855", 855 },
	{ "cp857", 857 },
	{ "cp858", 858 },
	{ "cp860", 860 },
	{ "cp861", 861 },
	{ "cp862", 862 },
	{ "cp863", 863 },
	{ "cp864", 864 },
	{ "cp865", 865 },
	{ "cp866", 866 },
	{ "cp869", 869 },
	{ "CP870", 870 },
	{ "CP871", 20871 },
	{ "cp875", 875 },
	{ "cp880", 20880 },
	{ "CP905", 20905 },
	// { "cp930", 50930 },
	// { "cp933", 50933 },
	// { "cp935", 50935 },
	// { "cp937", 50937 },
	// { "cp939", 50939 },
	{ "csASCII", 20127 },
	{ "csbig5", 950 },
	{ "csEUCKR", 51949 },
	{ "csEUCPkdFmtJapanese", 51932 },
	{ "csGB2312", 936 },
	{ "csGB231280", 936 },
	{ "csIBM037", 37 },
	{ "csIBM1026", 1026 },
	{ "csIBM273", 20273 },
	{ "csIBM277", 20277 },
	{ "csIBM278", 20278 },
	{ "csIBM280", 20280 },
	{ "csIBM284", 20284 },
	{ "csIBM285", 20285 },
	{ "csIBM290", 20290 },
	{ "csIBM297", 20297 },
	{ "csIBM420", 20420 },
	{ "csIBM423", 20423 },
	{ "csIBM424", 20424 },
	{ "csIBM500", 500 },
	{ "csIBM870", 870 },
	{ "csIBM871", 20871 },
	{ "csIBM880", 20880 },
	{ "csIBM905", 20905 },
	{ "csIBMThai", 20838 },
	{ "csISO2022JP", 50221 },
	{ "csISO2022KR", 50225 },
	{ "csISO58GB231280", 936 },
	{ "csISOLatin1", 28591 },
	{ "csISOLatin2", 28592 },
	{ "csISOLatin3", 28593 },
	{ "csISOLatin4", 28594 },
	{ "csISOLatin5", 28599 },
	{ "csISOLatin9", 28605 },
	{ "csISOLatinArabic", 28596 },
	{ "csISOLatinCyrillic", 28595 },
	{ "csISOLatinGreek", 28597 },
	{ "csISOLatinHebrew", 28598 },
	{ "csKOI8R", 20866 },
	{ "csKSC56011987", 949 },
	{ "csPC8CodePage437", 437 },
	{ "csShiftJIS", 932 },
	{ "csUnicode11UTF7", 65000 },
	{ "csWindows31J", 932 },
	{ "cyrillic", 28595 },
	{ "DIN_66003", 20106 },
	{ "DOS-720", 720 },
	{ "DOS-862", 862 },
	{ "DOS-874", 874 },
	{ "ebcdic-cp-ar1", 20420 },
	{ "ebcdic-cp-be", 500 },
	{ "ebcdic-cp-ca", 37 },
	{ "ebcdic-cp-ch", 500 },
	{ "EBCDIC-CP-DK", 20277 },
	{ "ebcdic-cp-es", 20284 },
	{ "ebcdic-cp-fi", 20278 },
	{ "ebcdic-cp-fr", 20297 },
	{ "ebcdic-cp-gb", 20285 },
	{ "ebcdic-cp-gr", 20423 },
	{ "ebcdic-cp-he", 20424 },
	{ "ebcdic-cp-is", 20871 },
	{ "ebcdic-cp-it", 20280 },
	{ "ebcdic-cp-nl", 37 },
	{ "EBCDIC-CP-NO", 20277 },
	{ "ebcdic-cp-roece", 870 },
	{ "ebcdic-cp-se", 20278 },
	{ "ebcdic-cp-tr", 20905 },
	{ "ebcdic-cp-us", 37 },
	{ "ebcdic-cp-wt", 37 },
	{ "ebcdic-cp-yu", 870 },
	{ "EBCDIC-Cyrillic", 20880 },
	{ "ebcdic-de-273+euro", 1141 },
	{ "ebcdic-dk-277+euro", 1142 },
	{ "ebcdic-es-284+euro", 1145 },
	{ "ebcdic-fi-278+euro", 1143 },
	{ "ebcdic-fr-297+euro", 1147 },
	{ "ebcdic-gb-285+euro", 1146 },
	{ "ebcdic-international-500+euro", 1148 },
	{ "ebcdic-is-871+euro", 1149 },
	{ "ebcdic-it-280+euro", 1144 },
	{ "EBCDIC-JP-kana", 20290 },
	{ "ebcdic-Latin9--euro", 20924 },
	{ "ebcdic-no-277+euro", 1142 },
	{ "ebcdic-se-278+euro", 1143 },
	{ "ebcdic-us-37+euro", 1140 },
	{ "ECMA-114", 28596 },
	{ "ECMA-118", 28597 },
	{ "ELOT_928", 28597 },
	{ "euc-cn", 51936 },
	{ "euc-jp", 51932 },
	{ "euc-kr", 51949 },
	{ "Extended_UNIX_Code_Packed_Format_for_Japanese", 51932 },
	{ "GB18030", 54936 },
	{ "GB2312", 936 },
	{ "GB2312-80", 936 },
	{ "GB231280", 936 },
	{ "GBK", 936 },
	{ "GB_2312-80", 936 },
	{ "German", 20106 },
	{ "greek", 28597 },
	{ "greek8", 28597 },
	{ "hebrew", 28598 },
	{ "hz-gb-2312", 52936 },
	{ "IBM-Thai", 20838 },
	{ "IBM00858", 858 },
	{ "IBM00924", 20924 },
	{ "IBM01047", 1047 },
	{ "IBM01140", 1140 },
	{ "IBM01141", 1141 },
	{ "IBM01142", 1142 },
	{ "IBM01143", 1143 },
	{ "IBM01144", 1144 },
	{ "IBM01145", 1145 },
	{ "IBM01146", 1146 },
	{ "IBM01147", 1147 },
	{ "IBM01148", 1148 },
	{ "IBM01149", 1149 },
	{ "IBM037", 37 },
	{ "IBM1026", 1026 },
	{ "IBM273", 20273 },
	{ "IBM277", 20277 },
	{ "IBM278", 20278 },
	{ "IBM280", 20280 },
	{ "IBM284", 20284 },
	{ "IBM285", 20285 },
	{ "IBM290", 20290 },
	{ "IBM297", 20297 },
	{ "IBM367", 20127 },
	{ "IBM420", 20420 },
	{ "IBM423", 20423 },
	{ "IBM424", 20424 },
	{ "IBM437", 437 },
	{ "IBM500", 500 },
	{ "ibm737", 737 },
	{ "ibm775", 775 },
	{ "ibm819", 28591 },
	{ "IBM850", 850 },
	{ "IBM852", 852 },
	{ "IBM855", 855 },
	{ "IBM857", 857 },
	{ "IBM860", 860 },
	{ "IBM861", 861 },
	{ "IBM862", 862 },
	{ "IBM863", 863 },
	{ "IBM864", 864 },
	{ "IBM865", 865 },
	{ "IBM866", 866 },
	{ "IBM869", 869 },
	{ "IBM870", 870 },
	{ "IBM871", 20871 },
	{ "IBM880", 20880 },
	{ "IBM905", 20905 },
	{ "irv", 20105 },
	{ "ISO-10646-UCS-2", 1200 },
	{ "iso-2022-jp", 50220 },
	{ "iso-2022-jpeuc", 51932 },
	{ "iso-2022-kr", 50225 },
	{ "iso-2022-kr-7", 50225 },
	{ "iso-2022-kr-7bit", 50225 },
	{ "iso-2022-kr-8", 51949 },
	{ "iso-2022-kr-8bit", 51949 },
	{ "iso-8859-1", 28591 },
	{ "iso-8859-11", 874 },
	{ "iso-8859-13", 28603 },
	{ "iso-8859-15", 28605 },
	{ "iso-8859-2", 28592 },
	{ "iso-8859-3", 28593 },
	{ "iso-8859-4", 28594 },
	{ "iso-8859-5", 28595 },
	{ "iso-8859-6", 28596 },
	{ "iso-8859-7", 28597 },
	{ "iso-8859-8", 28598 },
	{ "ISO-8859-8 Visual", 28598 },
	{ "iso-8859-8-i", 38598 },
	{ "iso-8859-9", 28599 },
	{ "iso-ir-100", 28591 },
	{ "iso-ir-101", 28592 },
	{ "iso-ir-109", 28593 },
	{ "iso-ir-110", 28594 },
	{ "iso-ir-126", 28597 },
	{ "iso-ir-127", 28596 },
	{ "iso-ir-138", 28598 },
	{ "iso-ir-144", 28595 },
	{ "iso-ir-148", 28599 },
	{ "iso-ir-149", 949 },
	{ "iso-ir-58", 936 },
	{ "iso-ir-6", 20127 },
	{ "ISO646-US", 20127 },
	{ "iso8859-1", 28591 },
	{ "iso8859-2", 28592 },
	{ "ISO_646.irv:1991", 20127 },
	{ "iso_8859-1", 28591 },
	{ "ISO_8859-15", 28605 },
	{ "iso_8859-1:1987", 28591 },
	{ "iso_8859-2", 28592 },
	{ "iso_8859-2:1987", 28592 },
	{ "ISO_8859-3", 28593 },
	{ "ISO_8859-3:1988", 28593 },
	{ "ISO_8859-4", 28594 },
	{ "ISO_8859-4:1988", 28594 },
	{ "ISO_8859-5", 28595 },
	{ "ISO_8859-5:1988", 28595 },
	{ "ISO_8859-6", 28596 },
	{ "ISO_8859-6:1987", 28596 },
	{ "ISO_8859-7", 28597 },
	{ "ISO_8859-7:1987", 28597 },
	{ "ISO_8859-8", 28598 },
	{ "ISO_8859-8:1988", 28598 },
	{ "ISO_8859-9", 28599 },
	{ "ISO_8859-9:1989", 28599 },
	{ "Johab", 1361 },
	{ "koi", 20866 },
	{ "koi8", 20866 },
	{ "koi8-r", 20866 },
	{ "koi8-ru", 21866 },
	{ "koi8-u", 21866 },
	{ "koi8r", 20866 },
	{ "korean", 949 },
	{ "ks-c-5601", 949 },
	{ "ks-c5601", 949 },
	{ "KSC5601", 949 },
	{ "KSC_5601", 949 },
	{ "ks_c_5601", 949 },
	{ "ks_c_5601-1987", 949 },
	{ "ks_c_5601-1989", 949 },
	{ "ks_c_5601_1987", 949 },
	{ "l1", 28591 },
	{ "l2", 28592 },
	{ "l3", 28593 },
	{ "l4", 28594 },
	{ "l5", 28599 },
	{ "l9", 28605 },
	{ "latin1", 28591 },
	{ "latin2", 28592 },
	{ "latin3", 28593 },
	{ "latin4", 28594 },
	{ "latin5", 28599 },
	{ "latin9", 28605 },
	{ "logical", 28598 },
	{ "macintosh", 10000 },
	{ "ms_Kanji", 932 },
	{ "Norwegian", 20108 },
	{ "NS_4551-1", 20108 },
	{ "PC-Multilingual-850+euro", 858 },
	{ "SEN_850200_B", 20107 },
	{ "shift-jis", 932 },
	{ "shift_jis", 932 },
	{ "sjis", 932 },
	{ "Swedish", 20107 },
	{ "TIS-620", 874 },
	{ "ucs-2", 1200 },
	{ "unicode", 1200 },
	{ "unicode-1-1-utf-7", 65000 },
	{ "unicode-1-1-utf-8", 65001 },
	{ "unicode-2-0-utf-7", 65000 },
	{ "unicode-2-0-utf-8", 65001 },
	// People get confused about the FFFE here.  We can't change this because it'd break existing apps.
	// This has been this way for a long time, including in Mlang.
	{ "unicodeFFFE", 1201 },             // Big Endian, BOM seems backwards, think of the BOM in little endian order.
	{ "us", 20127 },
	{ "us-ascii", 20127 },
	{ "utf-16", 1200 },
	{ "UTF-16BE", 1201 },
	{ "UTF-16LE", 1200 },
	{ "utf-32", 12000 },
	{ "UTF-32BE", 12001 },
	{ "UTF-32LE", 12000 },
	{ "utf-7", 65000 },
	{ "utf-8", 65001 },
	{ "visual", 28598 },
	{ "windows-1250", 1250 },
	{ "windows-1251", 1251 },
	{ "windows-1252", 1252 },
	{ "windows-1253", 1253 },
	{ "Windows-1254", 1254 },
	{ "windows-1255", 1255 },
	{ "windows-1256", 1256 },
	{ "windows-1257", 1257 },
	{ "windows-1258", 1258 },
	{ "windows-874", 874 },
	{ "x-ansi", 1252 },
	{ "x-Chinese-CNS", 20000 },
	{ "x-Chinese-Eten", 20002 },
	{ "x-cp1250", 1250 },
	{ "x-cp1251", 1251 },
	{ "x-cp20001", 20001 },
	{ "x-cp20003", 20003 },
	{ "x-cp20004", 20004 },
	{ "x-cp20005", 20005 },
	{ "x-cp20261", 20261 },
	{ "x-cp20269", 20269 },
	{ "x-cp20936", 20936 },
	{ "x-cp20949", 20949 },
	{ "x-cp50227", 50227 },
	// { "x-cp50229", 50229 },
	// { "X-EBCDIC-JapaneseAndUSCanada", 50931 },
	{ "X-EBCDIC-KoreanExtended", 20833 },
	{ "x-euc", 51932 },
	{ "x-euc-cn", 51936 },
	{ "x-euc-jp", 51932 },
	{ "x-Europa", 29001 },
	{ "x-IA5", 20105 },
	{ "x-IA5-German", 20106 },
	{ "x-IA5-Norwegian", 20108 },
	{ "x-IA5-Swedish", 20107 },
	{ "x-iscii-as", 57006 },
	{ "x-iscii-be", 57003 },
	{ "x-iscii-de", 57002 },
	{ "x-iscii-gu", 57010 },
	{ "x-iscii-ka", 57008 },
	{ "x-iscii-ma", 57009 },
	{ "x-iscii-or", 57007 },
	{ "x-iscii-pa", 57011 },
	{ "x-iscii-ta", 57004 },
	{ "x-iscii-te", 57005 },
	{ "x-mac-arabic", 10004 },
	{ "x-mac-ce", 10029 },
	{ "x-mac-chinesesimp", 10008 },
	{ "x-mac-chinesetrad", 10002 },
	{ "x-mac-croatian", 10082 },
	{ "x-mac-cyrillic", 10007 },
	{ "x-mac-greek", 10006 },
	{ "x-mac-hebrew", 10005 },
	{ "x-mac-icelandic", 10079 },
	{ "x-mac-japanese", 10001 },
	{ "x-mac-korean", 10003 },
	{ "x-mac-romanian", 10010 },
	{ "x-mac-thai", 10021 },
	{ "x-mac-turkish", 10081 },
	{ "x-mac-ukrainian", 10017 },
	{ "x-ms-cp932", 932 },
	{ "x-sjis", 932 },
	{ "x-unicode-1-1-utf-7", 65000 },
	{ "x-unicode-1-1-utf-8", 65001 },
	{ "x-unicode-2-0-utf-7", 65000 },
	{ "x-unicode-2-0-utf-8", 65001 },
	{ "x-x-big5", 950 },
};

// Returns the human-readable description of the encoding ( e.g. Hebrew (DOS)).
String REFEncoding::GetEncodingName() const
{
	// Calculate max entries in the name description table.
	int nMaxDesc = sizeof(s_CodePageDescTable) / sizeof(CP_NAME_DESC);

	// Search entire table to found first(only) item that CodePage matches.
	for (int i = 0; i < nMaxDesc; i ++)
	{
		if (s_CodePageDescTable[i].CodePage == m_nCodePage)
			// returns description name we found.
			return String(s_CodePageDescTable[i].DescName);
	}

	// Not found, just returns the empty string(NULL);
	return String();
}

// Gets or sets an codepage for the system's current ANSI code page.
// Different systems might use different codepage as the default.
int REFEncoding::GetDefaultCodePage()
{
	if (s_nDefaultCodePage == 0)
	{
		int nCodePage;
		// initialize the default code page.
#ifdef	_WIN32
		// NOTE: If we are a console program, we will need to use the GetConsoleOutputCP to instead GetACP function.
		nCodePage = (int)GetConsoleOutputCP();
		if (!nCodePage)
		{
			nCodePage = (int)GetACP();
		}
#else	//	!_WIN32
		String strName
#ifdef HAVE_LANGINFO_CODESET
			= nl_langinfo (CODESET);
#else	//	!HAVE_LANGINFO_CODESET
			= "iso-8859-1";
		// Try to get the used charset from environment variables.
		PCSTR pszLc = getenv("LC_ALL");
		if (!pszLc || !*pszLc)
		{
			pszLc = getenv("LC_CTYPE");
			if (!pszLc || !*pszLc)
				pszLc = getenv("LANG");
		}
		if (pszLc && *pszLc)
		{
			PCSTR pchDot = strchr(pszLc, '.');
			if (pchDot)
			{
				PCSTR pchMod = strchr(++pchDot, '@');
				if (!pchMod)
					pchMod = pchDot + strlen(pchDot);
				strName = String(pchDot, pchMod - pchDot);
			}
		}
#endif	//	!HAVE_LANGINFO_CODESET
		nCodePage = GetCodePageFromName(strName);
#endif	//	!_WIN32
		// Interlocked::Exchange(&s_nDefaultCodePage, nCodePage);
		s_nDefaultCodePage = nCodePage;
	}
	return s_nDefaultCodePage;
}

void REFEncoding::SetDefaultCodePage(int nCodePage)
{
	// TODO: Interlocked::Exchange(&s_nDefaultCodePage, nCodePage);
	s_nDefaultCodePage = nCodePage;
}

// Sets the default codepage name, the name can be NULL means detect codepage automatically.
void REFEncoding::SetDefaultCodePage(const char* pszName)
{
	int nCodePage = 0;
	if (pszName != NULL && *pszName != 0)
	{
		nCodePage = GetCodePageFromName(pszName);
		BFX_ENSURE1(nCodePage > 0, "Invalid codepage name specified.");
	}
	SetDefaultCodePage(nCodePage);
}

// Returns the correct code page number for this encoding name.
int REFEncoding::GetCodePageFromName(const char* pszName)
{
	typedef StringTraitsT<char> StrTraits;

	//
	// NOTE: The name is case-insensitive.
	//
	int nCodePage = 0;

	//
	// try looking it up in the raw data.
	//
	int nLeft = 0;
	int nRight = sizeof(s_pEncodingDataTable) / sizeof(CP_ENCODING_DATA);
	int nIndex;
	int nResult;

	if (pszName == NULL)
		goto Finish;

	// Binary search the array until we have only a couple of elements left and then just walk those elements.
	while ((nRight - nLeft) > 3)
	{
		nIndex = ((nRight - nLeft) / 2) + nLeft;

		nResult = StrTraits::StringCompareIgnore(pszName, s_pEncodingDataTable[nIndex].WebName);
		if (nResult == 0)
		{
			//We found the item, return the associated codepage.
			nCodePage = (s_pEncodingDataTable[nIndex].CodePage);
			goto Finish;
		}
		else if (nResult < 0)
			//The name that we're looking for is less than our current index.
			nRight = nIndex;
		else
			//The name that we're looking for is greater than our current index
			nLeft = nIndex;
	}

	// Walk the remaining elements (it'll be 3 or fewer).
	for (; nLeft <= nRight; nLeft ++)
	{
		if (StrTraits::StringCompareIgnore(pszName, s_pEncodingDataTable[nLeft].WebName) == 0)
		{
			nCodePage = s_pEncodingDataTable[nLeft].CodePage;
			goto Finish;
		}
	}
	// The encoding name is not valid.
	nCodePage = -1;

Finish:
	// TODO: At this point, Put the code page into our cached collection.
	return nCodePage;
}

// Returns an encoding object for a given name or a given code page value.
REFEncoding* REFEncoding::GetEncoding(const char* pszName)
{
	// attempt to load codepage from cached collection.
	int nCodePage = GetCodePageFromName(pszName);
	if (nCodePage == -1)	// invalid code page value.
		return NULL;

	return (GetEncoding(nCodePage));
}

// Initialize static resources.
void REFEncoding::StaticInitialize()
{
	if (s_pEncodings == NULL)
	{
		MutexHolder lock(&s_syncMutex);

		if (s_pEncodings == NULL)
		{
			s_pEncodings = new REFEncodingMap();
		}
	}
}

// Cleanup static resources, including shared memory, cached encoding objects, etc...
void REFEncoding::StaticFinalize()
{
	MutexHolder lock(&s_syncMutex);

	s_pEncodings = NULL;
}

// Returns the encoding associated with the specified code page identifier.
REFEncoding* REFEncoding::GetEncoding(int nCodePage)
{
	StaticInitialize();

	BFX_REQUIRE ((nCodePage >= 0 || nCodePage <= 65535) && "Invalid codepage specified.");

	REFEncoding*	pResult = NULL;

	// Pre-process default code page
	nCodePage = (nCodePage == CodePageDefault) ? GetDefaultCodePage() : nCodePage;

	// See if we have a cache table with our encoding in it already.
	if (!s_pEncodings->IsEmpty())
	{
		MutexHolder lock(&s_syncMutex);

		pResult = (*s_pEncodings)[nCodePage];
	}

	if (pResult == NULL)
	{
		MutexHolder lock(&s_syncMutex);

		// Double check that we don't have one in the table (in case another thread beat us here)
		if (!s_pEncodings->IsEmpty())
		{
			pResult = (*s_pEncodings)[nCodePage];
			if (pResult != NULL)
				return pResult;
		}

		// Special case the commonly used encoding classes here, then call
		// GetEncodingRare to avoid loading classes like MLangCodePageEncoding
		// and REFASCIIEncoding.  ASP.NET uses UTF-8 & ISO-8859-1.
		switch (nCodePage)
		{
		case CodePageDefault:					// 0, default code page
			BFX_ASSERT(false);	// Should never touch here.
			break;
		case CodePageUnicode:					// 1200, Unicode
			pResult = new REFUTF16Encoding();
			break;
		case CodePageBigEndian:					// 1201, big endian unicode
			pResult = new REFUTF16Encoding(true);
			break;
		case CodePageWindows1252:				// 1252, Windows
			pResult = new REFSBCSCodePageEncoding(nCodePage);
			break;
		case CodePageUTF8:						// 65001, UTF8
			pResult = new REFUTF8Encoding();
			break;

			// These are (hopefully) not very common, but also shouldn't slow us down much and make default
			// case able to handle more code pages by calling GetEncodingCodePage
		case CodePageNoOEM:						// 1
		case CodePageNoMac:						// 2
		case CodePageNoThread:					// 3
		case CodePageNoSymbol:					// 42
			// Win32 also allows the following special code page values.  We won't allow them except in the
			// CP_ACP case.
			// #define CP_ACP                    0           // default to ANSI code page
			// #define CP_OEMCP                  1           // default to OEM  code page
			// #define CP_MACCP                  2           // default to MAC  code page
			// #define CP_THREAD_ACP             3           // current thread's ANSI code page
			// #define CP_SYMBOL                 42          // SYMBOL translations
			BFX_ENSURE1(false, "not a supported code page.");
			break;

			// Have to do ASCII and Latin 1 first so they don't get loaded as code pages 
		case CodePageASCII:             // 20127
			pResult = new REFASCIIEncoding();
			break;
		case ISO_8859_1:                // 28591
			pResult = new REFLatin1Encoding();
			break;
		default:
			// 1st assume its a code page.
			pResult = GetEncodingCodePage(nCodePage);
			if (pResult == NULL)
				pResult = GetEncodingRare(nCodePage);
			break;
		}

		(*s_pEncodings)[nCodePage] = pResult;
	}

	return pResult;
}

REFEncoding* REFEncoding::GetEncodingCodePage(int nCodePage)
{
	REF<REFEncoding> pResult;

	// Single Byte or Double Byte Code Page? (0 if not found)
	int i = REFBaseCodePageEncoding::GetCodePageByteSize(nCodePage);
	if (i == 1)
		pResult = new REFSBCSCodePageEncoding(nCodePage);
	else if (i == 2)
		pResult = new REFDBCSCodePageEncoding(nCodePage);

	if (pResult != NULL)
		pResult->Autorelease();

	// Return null if we didn't find one.
	return pResult;
}

REFEncoding* REFEncoding::GetEncodingRare(int nCodePage)
{
	BFX_ASSERT(nCodePage != 0 && nCodePage != 1200 && nCodePage != 1201 && nCodePage != 65001);	// This code page isn't supported by GetEncodingRare!
	REF<REFEncoding> pResult;

	switch (nCodePage)
	{
#if	0
	case CodePageUTF7:              // 65000
		pResult = new UTF7Encoding();
		break;
#endif
	case CodePageUTF32:             // 12000
		pResult = new REFUTF32Encoding();
		break;
	case CodePageUTF32BE:           // 12001
		pResult = new REFUTF32Encoding(true);
		break;
#if	0
	case ISCIIAssemese:
	case ISCIIBengali:
	case ISCIIDevanagari:
	case ISCIIGujarathi:
	case ISCIIKannada:
	case ISCIIMalayalam:
	case ISCIIOriya:
	case ISCIIPanjabi:
	case ISCIITamil:
	case ISCIITelugu:
		pResult = new ISCIIEncoding(nCodePage);
		break;
		// GB2312-80 uses same code page for 20936 and mac 10008
#endif	// 0
	case CodePageMacGB2312:
		pResult = new REFDBCSCodePageEncoding(CodePageMacGB2312, CodePageGB2312);
		break;

		// Mac Korean 10003 and 20949 are the same
	case CodePageMacKorean:
		pResult = new REFDBCSCodePageEncoding(CodePageMacKorean, CodePageDLLKorean);
		break;
		// GB18030 Code Pages
#if	0
	case GB18030:
		pResult = new GB18030Encoding();
		break;
		// ISO2022 Code Pages
	case ISOKorean:
		//    case ISOSimplifiedCN
	case ChineseHZ:
	case ISO2022JP:         // JIS JP, full-width Katakana mode (no half-width Katakana)
	case ISO2022JPESC:      // JIS JP, esc sequence to do Katakana.
	case ISO2022JPSISO:     // JIS JP with Shift In/ Shift Out Katakana support
		pResult = new ISO2022Encoding(nCodePage);
		break;
#endif	// 0
	case DuplicateEUCCN:
	case ISOSimplifiedCN:
		pResult = new REFDBCSCodePageEncoding(nCodePage, EUCCN);    // Just maps to 936
		break;
#if	0
	case EUCJP:
		pResult = new EUCJPEncoding();
		break;
#endif	// 0
	case EUCKR:
		pResult = new REFDBCSCodePageEncoding(nCodePage, CodePageDLLKorean);    // Maps to 20949
		break;
	case ENC50229:
		BFX_ENSURE1(false, "The ISO-2022-CN Encoding (Code page 50229) is not supported.");
		break;
	case ISO_8859_8I:
		pResult = new REFSBCSCodePageEncoding(nCodePage, ISO_8859_8_Visual);        // Hebrew maps to a different code page
		break;
	default:
		// Not found, already tried codepage table code pages in GetEncoding()
		BFX_ENSURE1(false, "No data is available for encoding.");
		break;
	}

	if (pResult != NULL)
		pResult->Autorelease();

	return pResult;
}

// Calculates the number of characters produced by decoding a sequence of bytes.
int REFEncoding::GetCharCount(const BYTE* pBytes, int nCount) const
{
	return GetCharCount(pBytes, nCount, NULL);
}

// Decodes a sequence of bytes into a set of characters.
int REFEncoding::GetChars(const BYTE* pBytes, int nByteCount, OUT UCHAR* pChars, int nCharCount) const
{
	return GetChars(pBytes, nByteCount, pChars, nCharCount, NULL);
}

// Returns a Decoder object for this encoding. The returned object can be used to decode a sequence of bytes into a sequence of characters.
// Contrary to the GetChars family of methods, a Decoder can convert partial sequences of bytes into partial sequences of characters
// by maintaining the appropriate state between the conversions.
REFDecoder* REFEncoding::GetDecoder() const
{
	REFDecoder* pDecoder = new REFDecoder(this);
	pDecoder->Autorelease();
	return pDecoder;
}

REFDecoderFallback* REFEncoding::GetDecoderFallback() const
{
	return m_pDecoderFallback;
}

}	//	namespace BFX
